# Replication archive for: Coppock, Alexander and Seth J. Hill and Lynn Vavreck. 2020. "The Small Effects of Political Advertising are Small Regardless of Context, Message, Sender, or Receiver: Evidence from 59 Real-time Randomized Experiments" Science Advances, forthcoming.

library(tidyverse)

# manipulation check ------------------------------------------------------

# "We are confident treatments were delivered as intended: tiny fractions of the control groups claimed to have seen campaign ads (2 to 4%) compared with large majorities of the treatment groups (92 to 95%)."

exps <- read_rds("exps.rds")

exps <-
  exps %>%
  mutate(
    campaign_video = as.numeric(manip == 2),
    any_treat = as.numeric(assignment != "Control")
  ) %>%
  filter(wave >= 21)


exps %>%
  group_by(any_treat, wave) %>%
  summarise(mean(campaign_video, na.rm = TRUE))


# attrition check ---------------------------------------------------------

# Specifically, we conduct separate χ2 tests of the dependence between response and treatment assignment within each week of the study. Of the 29 tests, three return unadjusted p-values that are statistically significant. When we adjust for multiple comparisons using the Holm or Benjamini-Hochberg corrections (32, 33), none of the tests remain significant.

exps <- read_rds("exps_with_incompletes.rds")

exps <-
  exps %>%
  mutate(complete = as.numeric(!is.na(weights)))

exps %>% summarize(dropped = mean(complete == 0))

p_values <- 
  exps %>%
  split(.$wave) %>%
  map(~with(.,chisq.test(table(complete, Z)))) %>%
  map_dbl(~.$p.value)

p_holm <- p.adjust(p_values, method = "holm")
p_bh <- p.adjust(p_values, method = "BH")

sum(p_values <= 0.05)
sum(p_holm <= 0.05)
sum(p_bh <= 0.05)
